# -*- coding: utf-8 -*-
# @file: word2vec_preprocess.py
# @author: ZhuJiahui1991
# @time: 2018/2/1 22:24
# @version: v1.0

import numpy as np


def baike_process(read_filename, write_filename):

    with open(write_filename, 'w', encoding='utf-8') as fw:
        with open(read_filename, 'r', encoding='utf-8') as fr:
            for each_line in fr:
                this_line = each_line.replace(' ', '-')
                this_line = this_line.replace('\t|\t', ' ')
                fw.write(this_line.strip())
                fw.write('\n')


def filter_file_process(read_filename, write_filename):

    word_dict = dict()
    with open(write_filename, 'w', encoding='utf-8') as fw:
        with open(read_filename, 'r', encoding='utf-8') as fr:
            for each_line in fr:
                this_line = each_line.replace(' ', '-')
                this_line = this_line.replace('\t|\t', ' ').strip()

                line_splits = this_line.split()

                is_append = False
                for each_word in line_splits:
                    try:
                        this_word_value = word_dict[each_word]
                    except KeyError:
                        word_dict[each_word] = 0
                        is_append = True

                random_num = np.random.random()

                if is_append and random_num < 0.5:
                    fw.write(this_line)
                    fw.write('\n')

    print(len(word_dict))


def filter_char_process(read_filename, write_filename):

    char_dict = dict()
    with open(write_filename, 'w', encoding='utf-8') as fw:
        with open(read_filename, 'r', encoding='utf-8') as fr:
            for each_line in fr:
                this_line = each_line.replace('\t|\t', ' ').strip()
                this_char_list = []

                is_append = False
                for each_char in this_line:
                    if each_char not in [' ', '', '\t', '\n']:
                        this_char_list.append(each_char)
                        try:
                            this_word_value = char_dict[each_char]
                        except KeyError:
                            char_dict[each_char] = 0
                            is_append = True

                random_num = np.random.random()

                if is_append and len(this_char_list) > 2 and random_num < 1.0:
                    fw.write(' '.join(this_char_list))
                    fw.write('\n')

    print(len(char_dict))


def baike_process_test():
    read_filename = 'D:/Share/Dataset/baike-segment.txt'
    write_filename = 'D:/Share/Dataset/processed-baike-segment.txt'
    baike_process(read_filename, write_filename)


def baike_process_test1():
    read_filename = 'D:/Share/Dataset/baike-segment.txt'
    write_filename = 'D:/Share/Dataset/filtered-baike-segment.txt'
    filter_file_process(read_filename, write_filename)


def baike_process_test2():
    read_filename = 'D:/Share/Dataset/baike-segment.txt'
    write_filename = 'D:/Share/Dataset/char-baike-segment.txt'
    filter_char_process(read_filename, write_filename)


def sohu_process_test1():
    read_filename = 'D:/Share/Dataset/sohu-segment.txt'
    write_filename = 'D:/Share/Dataset/filtered-sohu-segment.txt'
    filter_file_process(read_filename, write_filename)


def sohu_process_test2():
    read_filename = 'D:/Share/Dataset/sohu-segment.txt'
    write_filename = 'D:/Share/Dataset/char-sohu-segment.txt'
    filter_char_process(read_filename, write_filename)


def all_preprocess():
    read_file_list = ['D:/Share/Dataset/filtered-baike-segment.txt',
                      'D:/Share/Dataset/char-baike-segment.txt',
                      'D:/Share/Dataset/filtered-sohu-segment.txt',
                      'D:/Share/Dataset/char-sohu-segment.txt']
    write_filename = 'D:/Share/Dataset/all-segment.txt'

    with open(write_filename, 'w', encoding='utf-8') as fw:
        for each_file in read_file_list:
            with open(each_file, 'r', encoding='utf-8') as fr:
                for each_line in fr:
                    fw.write(each_line.strip())
                    fw.write('\n')


if __name__ == "__main__":
    # baike_process_test2()
    # sohu_process_test1()
    # sohu_process_test2()
    all_preprocess()
